{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Watch the Trained Agent"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.Start the Environment for Trained Agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:unityagents:\n",
      "'Academy' started successfully!\n",
      "Unity Academy name: Academy\n",
      "        Number of Brains: 1\n",
      "        Number of External Brains : 1\n",
      "        Lesson number : 0\n",
      "        Reset Parameters :\n",
      "\t\t\n",
      "Unity brain name: CrawlerBrain\n",
      "        Number of Visual Observations (per agent): 0\n",
      "        Vector Observation space type: continuous\n",
      "        Vector Observation space size (per agent): 129\n",
      "        Number of stacked Vector Observation: 1\n",
      "        Vector Action space type: continuous\n",
      "        Vector Action space size (per agent): 20\n",
      "        Vector Action descriptions: , , , , , , , , , , , , , , , , , , , \n"
     ]
    }
   ],
   "source": [
    "from unityagents import UnityEnvironment\n",
    "import numpy as np\n",
    "import torch\n",
    "from ppo_agent import Agent\n",
    "\n",
    "env = UnityEnvironment(file_name='Crawler.exe')\n",
    "agent_1024 = Agent(state_size=129, action_size=20, random_seed=8, n_agent=12, fc1_units=1024, fc2_units=1024)\n",
    "agent_128 = Agent(state_size=129, action_size=20, random_seed=8, n_agent=12, fc1_units=128, fc2_units=128)\n",
    "\n",
    "# get the default brain\n",
    "brain_name = env.brain_names[0]\n",
    "brain = env.brains[brain_name]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Prepare Player and Load Weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def play(agent, episodes=5, max_t=1000):\n",
    "    action_size=20\n",
    "    n_agents=12\n",
    "    t_max = 0\n",
    "    for i_episode in range(episodes):\n",
    "        env_info = env.reset(train_mode=False)[brain_name]     # reset the environment    \n",
    "        states = env_info.vector_observations                  # get the current state (for each agent)\n",
    "        agent_scores = np.zeros(n_agents)\n",
    "        for t in range(max_t):    \n",
    "            actions, log_probs, _, values = agent.act(states)\n",
    "            env_info = env.step(actions)[brain_name]           # send all actions to tne environment\n",
    "            next_states = env_info.vector_observations         # get next state (for each agent)\n",
    "            rewards = env_info.rewards                         # get reward (for each agent)\n",
    "            dones = np.array([1 if t else 0 for t in env_info.local_done])\n",
    "            agent.save_step([states, values.detach(), actions, log_probs.detach(), rewards, 1 - dones])\n",
    "            agent_scores += env_info.rewards                         # update the score (for each agent)\n",
    "            states = next_states                               # roll over states to next time step\n",
    "            t_max = t\n",
    "            if np.any(dones):                                  # exit loop if episode finished\n",
    "                break\n",
    "        print('Episode: {}, Average Score (over agents): {:.2f}, Max Timestep: {} '\\\n",
    "              .format(i_episode, np.mean(agent_scores), t_max)) \n",
    "              "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Play Before  Training, Trained with Max Step = 1000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode: 0, Average Score (over agents): 14.45, Max Timestep: 299 \n",
      "Episode: 1, Average Score (over agents): 15.26, Max Timestep: 299 \n",
      "Episode: 2, Average Score (over agents): 16.00, Max Timestep: 299 \n"
     ]
    }
   ],
   "source": [
    "play(agent=agent_1024, episodes=3, max_t=300)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4. Prepare Loader "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load(agent, actor_file, critic_file):\n",
    "    agent.actor_critic.actor.load_state_dict(torch.load(actor_file))\n",
    "    agent.actor_critic.critic.load_state_dict(torch.load(critic_file))     "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5. Load and Play:  Trained Weights with Max Step = 1000,  Score = 500, agent_1024"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode: 0, Average Score (over agents): 22.43, Max Timestep: 54 \n",
      "Episode: 1, Average Score (over agents): 105.05, Max Timestep: 188 \n",
      "Episode: 2, Average Score (over agents): 19.97, Max Timestep: 61 \n"
     ]
    }
   ],
   "source": [
    "load(agent_1024, 'checkpoints/checkpoint_actor_676epis_500score.pth', 'checkpoints/checkpoint_critic_676epis_500score.pth') \n",
    "play(agent_1024, episodes=3, max_t=1000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 6. Load and Play: Trained Weights with Max Step = 1500, Score = 650, agent_1024"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode: 0, Average Score (over agents): 427.84, Max Timestep: 867 \n",
      "Episode: 1, Average Score (over agents): 196.80, Max Timestep: 399 \n",
      "Episode: 2, Average Score (over agents): 441.65, Max Timestep: 901 \n"
     ]
    }
   ],
   "source": [
    "load(agent_1024, 'checkpoints/checkpoint_actor_550epis_650score.pth', 'checkpoints/checkpoint_critic_550epis_650score.pth')  \n",
    "play(agent_1024, episodes=3, max_t=1500)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 7. Load and Play: Trained Weights with Max Step = 2000, Score = 800, agent_128"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode: 0, Average Score (over agents): 147.95, Max Timestep: 234 \n",
      "Episode: 1, Average Score (over agents): 605.50, Max Timestep: 943 \n",
      "Episode: 2, Average Score (over agents): 576.79, Max Timestep: 914 \n"
     ]
    }
   ],
   "source": [
    "load(agent_128, 'checkpoints/checkpoint_actor_532epis_800score.pth', 'checkpoints/checkpoint_critic_532epis_800score.pth')  \n",
    "play(agent_128, episodes=3, max_t=2000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 8. Load and Play: Trained Weights with Max Step = 2000, Score = 1200, agent_128"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode: 0, Average Score (over agents): 80.24, Max Timestep: 117 \n",
      "Episode: 1, Average Score (over agents): 81.68, Max Timestep: 118 \n",
      "Episode: 2, Average Score (over agents): 19.43, Max Timestep: 40 \n"
     ]
    }
   ],
   "source": [
    "load(agent_128, 'checkpoints/checkpoint_actor_678epis_1200score.pth', 'checkpoints/checkpoint_critic_678epis_1200score.pth')  \n",
    "play(agent_128, episodes=3, max_t=2000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:Anaconda_3]",
   "language": "python",
   "name": "conda-env-Anaconda_3-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
